{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"Multiple Model.ipynb","provenance":[],"collapsed_sections":["uYM5MfCr869v","MTqVNbVULB9G","PK3Qj5GPHLRW","s11gvUsLmjV5"],"toc_visible":true,"authorship_tag":"ABX9TyParFUaMDIvAuqLuEeblDx0"},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU"},"cells":[{"cell_type":"markdown","metadata":{"id":"uYM5MfCr869v","colab_type":"text"},"source":["# 一、数据读取和格式化"]},{"cell_type":"code","metadata":{"id":"QiW4MCHX8oE6","colab_type":"code","outputId":"205ffd26-9632-4c2b-a9eb-054e68f580ea","executionInfo":{"status":"ok","timestamp":1585568680124,"user_tz":-480,"elapsed":26828,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}},"colab":{"base_uri":"https://localhost:8080/","height":122}},"source":["from google.colab import drive\n","drive.mount('/content/drive')\n","import pandas as pd"],"execution_count":2,"outputs":[{"output_type":"stream","text":["Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n","\n","Enter your authorization code:\n","··········\n","Mounted at /content/drive\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"YAn-T5EZ9TUt","colab_type":"code","outputId":"d154aec1-cc88-4455-faf0-06f35fc4cf34","executionInfo":{"status":"ok","timestamp":1585568681537,"user_tz":-480,"elapsed":28222,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}},"colab":{"base_uri":"https://localhost:8080/","height":204}},"source":["path=\"/content/drive/My Drive/Colab Notebooks/NLP/06_Multiple Model/\"\n","df=pd.read_csv(path+\"tweets_0.80.csv\")\n","df.head()"],"execution_count":3,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>Unnamed: 0</th>\n","      <th>target</th>\n","      <th>id</th>\n","      <th>date</th>\n","      <th>time</th>\n","      <th>username</th>\n","      <th>tweet</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>0</td>\n","      <td>4</td>\n","      <td>1238978214090792960</td>\n","      <td>2020-03-14</td>\n","      <td>23:59:59</td>\n","      <td>ok32650586</td>\n","      <td>I hope everything turns ok! Sending love and p...</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>28</td>\n","      <td>0</td>\n","      <td>1238281456918482944</td>\n","      <td>2020-03-13</td>\n","      <td>01:51:20</td>\n","      <td>ok32650586</td>\n","      <td>Nobody announced that. We know they can be inf...</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>46</td>\n","      <td>4</td>\n","      <td>1238048212822274048</td>\n","      <td>2020-03-12</td>\n","      <td>10:24:30</td>\n","      <td>ok32650586</td>\n","      <td>Organic pumpkin is the best for their digestiv...</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>48</td>\n","      <td>4</td>\n","      <td>1237965669649412097</td>\n","      <td>2020-03-12</td>\n","      <td>04:56:30</td>\n","      <td>ok32650586</td>\n","      <td>Wow !! They are BIG dogs like you sweetie pie!...</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>59</td>\n","      <td>0</td>\n","      <td>1237725578196742144</td>\n","      <td>2020-03-11</td>\n","      <td>13:02:28</td>\n","      <td>ok32650586</td>\n","      <td>I guess we will have to tell our simple minded...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["   Unnamed: 0  ...                                              tweet\n","0           0  ...  I hope everything turns ok! Sending love and p...\n","1          28  ...  Nobody announced that. We know they can be inf...\n","2          46  ...  Organic pumpkin is the best for their digestiv...\n","3          48  ...  Wow !! They are BIG dogs like you sweetie pie!...\n","4          59  ...  I guess we will have to tell our simple minded...\n","\n","[5 rows x 7 columns]"]},"metadata":{"tags":[]},"execution_count":3}]},{"cell_type":"code","metadata":{"id":"GzRFFcxS9lf-","colab_type":"code","outputId":"de207d6d-2394-4fd5-fd38-b5552c3e763d","executionInfo":{"status":"ok","timestamp":1585568681539,"user_tz":-480,"elapsed":28212,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}},"colab":{"base_uri":"https://localhost:8080/","height":68}},"source":["df.target.value_counts()"],"execution_count":4,"outputs":[{"output_type":"execute_result","data":{"text/plain":["4    80032\n","0    55147\n","Name: target, dtype: int64"]},"metadata":{"tags":[]},"execution_count":4}]},{"cell_type":"code","metadata":{"id":"F5MeGE6B-Bk3","colab_type":"code","colab":{}},"source":["#正负样本均采样20000条\n","data_pos=df.loc[df.target==4].sample(n=20000, random_state=2020)\n","data_neg=df.loc[df.target==0].sample(n=20000, random_state=2020)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"ft7tHRwu-iJ7","colab_type":"code","outputId":"2bfe1f3f-6bcc-4c0b-a849-11562d4c9441","executionInfo":{"status":"ok","timestamp":1585568681541,"user_tz":-480,"elapsed":28198,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}},"colab":{"base_uri":"https://localhost:8080/","height":68}},"source":["#合并正负样本\n","data=pd.concat([data_pos,data_neg])\n","data.target.value_counts()"],"execution_count":6,"outputs":[{"output_type":"execute_result","data":{"text/plain":["4    20000\n","0    20000\n","Name: target, dtype: int64"]},"metadata":{"tags":[]},"execution_count":6}]},{"cell_type":"code","metadata":{"id":"FQ750PBV-_rS","colab_type":"code","outputId":"050cd036-4adf-451c-d948-695f964f5e65","executionInfo":{"status":"ok","timestamp":1585568681542,"user_tz":-480,"elapsed":28091,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}},"colab":{"base_uri":"https://localhost:8080/","height":272}},"source":["#删除第一列\n","data.drop(columns=data.columns[0],inplace=True)\n","#删除多余列\n","data.drop(columns=data.columns[1:-1],inplace=True)\n","#正向标签设置为1\n","data.loc[data['target'] ==4, 'target'] = 1\n","print(data.target.value_counts())\n","print(data['tweet'][0])\n","#重采样--相当于shuffle\n","data=data.sample(frac=1,random_state=2020)\n","data.head()"],"execution_count":7,"outputs":[{"output_type":"stream","text":["1    20000\n","0    20000\n","Name: target, dtype: int64\n","I hope everything turns ok! Sending love and prayers for a good outcome. 🙏❤️🐶❤️🌹\n"],"name":"stdout"},{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>target</th>\n","      <th>tweet</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>102850</th>\n","      <td>1</td>\n","      <td>When something pops up I can afford and it’s a...</td>\n","    </tr>\n","    <tr>\n","      <th>95845</th>\n","      <td>0</td>\n","      <td>Nah it’s just a joke, only Dick(Ric) and Donna...</td>\n","    </tr>\n","    <tr>\n","      <th>58554</th>\n","      <td>1</td>\n","      <td>Donovan mitchell has rona lmao the season is s...</td>\n","    </tr>\n","    <tr>\n","      <th>65305</th>\n","      <td>0</td>\n","      <td>A potential Evans and Lingard backfield in 202...</td>\n","    </tr>\n","    <tr>\n","      <th>87962</th>\n","      <td>0</td>\n","      <td>Here’s the guy. Mike Minor. Seriously. Greinke...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["        target                                              tweet\n","102850       1  When something pops up I can afford and it’s a...\n","95845        0  Nah it’s just a joke, only Dick(Ric) and Donna...\n","58554        1  Donovan mitchell has rona lmao the season is s...\n","65305        0  A potential Evans and Lingard backfield in 202...\n","87962        0  Here’s the guy. Mike Minor. Seriously. Greinke..."]},"metadata":{"tags":[]},"execution_count":7}]},{"cell_type":"markdown","metadata":{"id":"MTqVNbVULB9G","colab_type":"text"},"source":["# 二、数据预处理及划分"]},{"cell_type":"code","metadata":{"id":"_ChqEyLwNV8K","colab_type":"code","outputId":"5e98e619-ab6b-471d-8356-f353dacac8ee","executionInfo":{"status":"ok","timestamp":1585568682594,"user_tz":-480,"elapsed":29130,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}},"colab":{"base_uri":"https://localhost:8080/","height":68}},"source":["import nltk\n","nltk.download('stopwords')"],"execution_count":8,"outputs":[{"output_type":"stream","text":["[nltk_data] Downloading package stopwords to /root/nltk_data...\n","[nltk_data]   Unzipping corpora/stopwords.zip.\n"],"name":"stdout"},{"output_type":"execute_result","data":{"text/plain":["True"]},"metadata":{"tags":[]},"execution_count":8}]},{"cell_type":"code","metadata":{"id":"r3iE_Il-LRP2","colab_type":"code","outputId":"239afcfd-31f3-434b-9114-87a9b2ff9f97","executionInfo":{"status":"ok","timestamp":1585568684674,"user_tz":-480,"elapsed":31200,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}},"colab":{"base_uri":"https://localhost:8080/","height":204}},"source":["# Removing Twitter Handles (@user)\n","data['clean_tweet'] = data['tweet'].str.replace(\"@\", \"\") \n","# Removing links\n","data['clean_tweet'] = data['tweet'].str.replace(r\"http\\S+\", \"\") \n","# Removing Punctuations, Numbers, and Special Characters\n","data['clean_tweet'] = data['tweet'].str.replace(\"[^a-zA-Z]\", \" \") \n","# Remove stop words\n","import nltk\n","stopwords=nltk.corpus.stopwords.words('english')\n","def remove_stopwords(text):\n","    clean_text=' '.join([word for word in text.split() if word not in stopwords])\n","    return clean_text\n","data['clean_tweet'] = data['tweet'].apply(lambda text : remove_stopwords(text.lower()))\n","data.head()"],"execution_count":9,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>target</th>\n","      <th>tweet</th>\n","      <th>clean_tweet</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>102850</th>\n","      <td>1</td>\n","      <td>When something pops up I can afford and it’s a...</td>\n","      <td>something pops afford it’s good idea go (cause...</td>\n","    </tr>\n","    <tr>\n","      <th>95845</th>\n","      <td>0</td>\n","      <td>Nah it’s just a joke, only Dick(Ric) and Donna...</td>\n","      <td>nah it’s joke, dick(ric) donna listed list betray</td>\n","    </tr>\n","    <tr>\n","      <th>58554</th>\n","      <td>1</td>\n","      <td>Donovan mitchell has rona lmao the season is s...</td>\n","      <td>donovan mitchell rona lmao season done. suspen...</td>\n","    </tr>\n","    <tr>\n","      <th>65305</th>\n","      <td>0</td>\n","      <td>A potential Evans and Lingard backfield in 202...</td>\n","      <td>potential evans lingard backfield 2021 terrifying</td>\n","    </tr>\n","    <tr>\n","      <th>87962</th>\n","      <td>0</td>\n","      <td>Here’s the guy. Mike Minor. Seriously. Greinke...</td>\n","      <td>here’s guy. mike minor. seriously. greinke mig...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["        target  ...                                        clean_tweet\n","102850       1  ...  something pops afford it’s good idea go (cause...\n","95845        0  ...  nah it’s joke, dick(ric) donna listed list betray\n","58554        1  ...  donovan mitchell rona lmao season done. suspen...\n","65305        0  ...  potential evans lingard backfield 2021 terrifying\n","87962        0  ...  here’s guy. mike minor. seriously. greinke mig...\n","\n","[5 rows x 3 columns]"]},"metadata":{"tags":[]},"execution_count":9}]},{"cell_type":"code","metadata":{"id":"Zf7AE4q9G_4D","colab_type":"code","colab":{}},"source":["#划分训练集和测试集\n","from sklearn.model_selection import train_test_split\n","X_train, X_test, y_train, y_test = train_test_split( data[\"clean_tweet\"], data[\"target\"], test_size=0.2, random_state=42)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"fse3lVlQmrWj","colab_type":"code","colab":{}},"source":["data.to_csv(path+\"data_tweet_clean.csv\",index=False)"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"PK3Qj5GPHLRW","colab_type":"text"},"source":["# 三、Baseline模型"]},{"cell_type":"code","metadata":{"id":"nmNuXrBYHIzh","colab_type":"code","colab":{}},"source":["from sklearn.metrics import accuracy_score\n","from sklearn.linear_model import LogisticRegression \n","from sklearn.feature_extraction.text import CountVectorizer\n","from xgboost import XGBClassifier"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"9DJkPALTN9ym","colab_type":"code","outputId":"858d9574-aec5-4224-8baa-fecc11c40e8f","executionInfo":{"status":"ok","timestamp":1585378184467,"user_tz":-480,"elapsed":6313,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"source":["count_vectorizer = CountVectorizer(stop_words='english') \n","cv = count_vectorizer.fit_transform(data['clean_tweet'])\n","cv.shape"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(40000, 38005)"]},"metadata":{"tags":[]},"execution_count":51}]},{"cell_type":"code","metadata":{"id":"7U4ZBRgUOU8w","colab_type":"code","colab":{}},"source":["baseX_train,baseX_test,base_y_train,base_y_test = train_test_split(cv , data['target'] , test_size=.2, random_state=42)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"_xxMSmuqOrmt","colab_type":"code","outputId":"9977211e-27e6-4d3b-ed44-0bed8daa6829","executionInfo":{"status":"ok","timestamp":1585378185711,"user_tz":-480,"elapsed":7498,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}},"colab":{"base_uri":"https://localhost:8080/","height":51}},"source":["# LG\n","lg = LogisticRegression(C=0.3,max_iter=2000)\n","lg.fit(baseX_train,base_y_train)\n","print(\"train accuracy_score\",lg.score(baseX_train,base_y_train))\n","prediction_lg = lg.predict(baseX_test)\n","print(\"test accuracy_score\",accuracy_score(prediction_lg,base_y_test))"],"execution_count":0,"outputs":[{"output_type":"stream","text":["train accuracy_score 0.9195\n","test accuracy_score 0.831875\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"QfbhgJBRjYyQ","colab_type":"code","outputId":"b47cba14-a67a-4247-c30b-9b6df45c5aa4","executionInfo":{"status":"ok","timestamp":1585378509500,"user_tz":-480,"elapsed":331204,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}},"colab":{"base_uri":"https://localhost:8080/","height":51}},"source":["# XGBC\n","xgbc = XGBClassifier(max_depth=6, n_estimators=3000, nthread= 3)\n","xgbc.fit(baseX_train,base_y_train)\n","print(\"train accuracy_score\",xgbc.score(baseX_train,base_y_train))\n","prediction_xgb = xgbc.predict(baseX_test)\n","print(\"test accuracy_score\",accuracy_score(prediction_xgb,base_y_test))"],"execution_count":0,"outputs":[{"output_type":"stream","text":["train accuracy_score 0.910625\n","test accuracy_score 0.832625\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"lIELd4h_j-TY","colab_type":"text"},"source":["# 四、多模型的训练（选取进行融合的模型）"]},{"cell_type":"markdown","metadata":{"id":"Fg5L63hONd1r","colab_type":"text"},"source":["设置进行融合的模型效果均比baseline模型的性能好  \n","train acc>0.920  \n","test acc>0.833\n","\n","如果不满足条件则认为该模型的性能不足以进行融合"]},{"cell_type":"markdown","metadata":{"id":"BJBBECnFkUyR","colab_type":"text"},"source":["## 4.1 Helper function"]},{"cell_type":"code","metadata":{"id":"ZgqQQ9anj9MD","colab_type":"code","colab":{}},"source":["#文本的编码\n","def text_encode(texts, tokenizer, max_len=512):\n","    all_tokens = []\n","    all_masks = []\n","    all_segments = []\n","    print(texts)\n","    for text in texts:\n","        text = tokenizer.tokenize(text)\n","            \n","        text = text[:max_len-2]\n","        input_sequence = [\"[CLS]\"] + text + [\"[SEP]\"]\n","        pad_len = max_len - len(input_sequence)\n","        \n","        tokens = tokenizer.convert_tokens_to_ids(input_sequence)\n","        tokens += [0] * pad_len\n","        pad_masks = [1] * len(input_sequence) + [0] * pad_len\n","        segment_ids = [0] * max_len\n","        \n","        all_tokens.append(tokens)\n","        all_masks.append(pad_masks)\n","        all_segments.append(segment_ids)\n","    \n","    return np.array(all_tokens), np.array(all_masks), np.array(all_segments)\n","\n","#预训练模型的构建\n","def build_model(encoder_layer, max_len=512):\n","    input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"input_word_ids\")\n","    input_mask = Input(shape=(max_len,), dtype=tf.int32, name=\"input_mask\")\n","    segment_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"segment_ids\")\n","\n","    _, sequence_output = encoder_layer([input_word_ids, input_mask, segment_ids])\n","    clf_output = sequence_output[:, 0, :]\n","    out = Dense(2, activation='sigmoid')(clf_output)\n","    \n","    model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)\n","    model.compile(Adam(lr=2e-6), loss='sparse_categorical_crossentropy', metrics=['sparse_categorical_accuracy'])\n","    \n","    return model"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"6qn371Riw8yp","colab_type":"code","outputId":"4a104b6b-b204-476f-c476-758faef0c442","executionInfo":{"status":"ok","timestamp":1585568722307,"user_tz":-480,"elapsed":9167,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}},"colab":{"base_uri":"https://localhost:8080/","height":122}},"source":["!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py\n","!pip install sentencepiece\n","import tokenization"],"execution_count":12,"outputs":[{"output_type":"stream","text":["Collecting sentencepiece\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/74/f4/2d5214cbf13d06e7cb2c20d84115ca25b53ea76fa1f0ade0e3c9749de214/sentencepiece-0.1.85-cp36-cp36m-manylinux1_x86_64.whl (1.0MB)\n","\r\u001b[K     |▎                               | 10kB 29.4MB/s eta 0:00:01\r\u001b[K     |▋                               | 20kB 3.0MB/s eta 0:00:01\r\u001b[K     |█                               | 30kB 4.0MB/s eta 0:00:01\r\u001b[K     |█▎                              | 40kB 2.9MB/s eta 0:00:01\r\u001b[K     |█▋                              | 51kB 3.4MB/s eta 0:00:01\r\u001b[K     |██                              | 61kB 4.0MB/s eta 0:00:01\r\u001b[K     |██▏                             | 71kB 4.3MB/s eta 0:00:01\r\u001b[K     |██▌                             | 81kB 4.4MB/s eta 0:00:01\r\u001b[K     |██▉                             | 92kB 4.9MB/s eta 0:00:01\r\u001b[K     |███▏                            | 102kB 4.8MB/s eta 0:00:01\r\u001b[K     |███▌                            | 112kB 4.8MB/s eta 0:00:01\r\u001b[K     |███▉                            | 122kB 4.8MB/s eta 0:00:01\r\u001b[K     |████                            | 133kB 4.8MB/s eta 0:00:01\r\u001b[K     |████▍                           | 143kB 4.8MB/s eta 0:00:01\r\u001b[K     |████▊                           | 153kB 4.8MB/s eta 0:00:01\r\u001b[K     |█████                           | 163kB 4.8MB/s eta 0:00:01\r\u001b[K     |█████▍                          | 174kB 4.8MB/s eta 0:00:01\r\u001b[K     |█████▊                          | 184kB 4.8MB/s eta 0:00:01\r\u001b[K     |██████                          | 194kB 4.8MB/s eta 0:00:01\r\u001b[K     |██████▎                         | 204kB 4.8MB/s eta 0:00:01\r\u001b[K     |██████▋                         | 215kB 4.8MB/s eta 0:00:01\r\u001b[K     |███████                         | 225kB 4.8MB/s eta 0:00:01\r\u001b[K     |███████▎                        | 235kB 4.8MB/s eta 0:00:01\r\u001b[K     |███████▋                        | 245kB 4.8MB/s eta 0:00:01\r\u001b[K     |███████▉                        | 256kB 4.8MB/s eta 0:00:01\r\u001b[K     |████████▏                       | 266kB 4.8MB/s eta 0:00:01\r\u001b[K     |████████▌                       | 276kB 4.8MB/s eta 0:00:01\r\u001b[K     |████████▉                       | 286kB 4.8MB/s eta 0:00:01\r\u001b[K     |█████████▏                      | 296kB 4.8MB/s eta 0:00:01\r\u001b[K     |█████████▌                      | 307kB 4.8MB/s eta 0:00:01\r\u001b[K     |█████████▊                      | 317kB 4.8MB/s eta 0:00:01\r\u001b[K     |██████████                      | 327kB 4.8MB/s eta 0:00:01\r\u001b[K     |██████████▍                     | 337kB 4.8MB/s eta 0:00:01\r\u001b[K     |██████████▊                     | 348kB 4.8MB/s eta 0:00:01\r\u001b[K     |███████████                     | 358kB 4.8MB/s eta 0:00:01\r\u001b[K     |███████████▍                    | 368kB 4.8MB/s eta 0:00:01\r\u001b[K     |███████████▋                    | 378kB 4.8MB/s eta 0:00:01\r\u001b[K     |████████████                    | 389kB 4.8MB/s eta 0:00:01\r\u001b[K     |████████████▎                   | 399kB 4.8MB/s eta 0:00:01\r\u001b[K     |████████████▋                   | 409kB 4.8MB/s eta 0:00:01\r\u001b[K     |█████████████                   | 419kB 4.8MB/s eta 0:00:01\r\u001b[K     |█████████████▎                  | 430kB 4.8MB/s eta 0:00:01\r\u001b[K     |█████████████▌                  | 440kB 4.8MB/s eta 0:00:01\r\u001b[K     |█████████████▉                  | 450kB 4.8MB/s eta 0:00:01\r\u001b[K     |██████████████▏                 | 460kB 4.8MB/s eta 0:00:01\r\u001b[K     |██████████████▌                 | 471kB 4.8MB/s eta 0:00:01\r\u001b[K     |██████████████▉                 | 481kB 4.8MB/s eta 0:00:01\r\u001b[K     |███████████████▏                | 491kB 4.8MB/s eta 0:00:01\r\u001b[K     |███████████████▍                | 501kB 4.8MB/s eta 0:00:01\r\u001b[K     |███████████████▊                | 512kB 4.8MB/s eta 0:00:01\r\u001b[K     |████████████████                | 522kB 4.8MB/s eta 0:00:01\r\u001b[K     |████████████████▍               | 532kB 4.8MB/s eta 0:00:01\r\u001b[K     |████████████████▊               | 542kB 4.8MB/s eta 0:00:01\r\u001b[K     |█████████████████               | 552kB 4.8MB/s eta 0:00:01\r\u001b[K     |█████████████████▎              | 563kB 4.8MB/s eta 0:00:01\r\u001b[K     |█████████████████▋              | 573kB 4.8MB/s eta 0:00:01\r\u001b[K     |██████████████████              | 583kB 4.8MB/s eta 0:00:01\r\u001b[K     |██████████████████▎             | 593kB 4.8MB/s eta 0:00:01\r\u001b[K     |██████████████████▋             | 604kB 4.8MB/s eta 0:00:01\r\u001b[K     |███████████████████             | 614kB 4.8MB/s eta 0:00:01\r\u001b[K     |███████████████████▏            | 624kB 4.8MB/s eta 0:00:01\r\u001b[K     |███████████████████▌            | 634kB 4.8MB/s eta 0:00:01\r\u001b[K     |███████████████████▉            | 645kB 4.8MB/s eta 0:00:01\r\u001b[K     |████████████████████▏           | 655kB 4.8MB/s eta 0:00:01\r\u001b[K     |████████████████████▌           | 665kB 4.8MB/s eta 0:00:01\r\u001b[K     |████████████████████▉           | 675kB 4.8MB/s eta 0:00:01\r\u001b[K     |█████████████████████▏          | 686kB 4.8MB/s eta 0:00:01\r\u001b[K     |█████████████████████▍          | 696kB 4.8MB/s eta 0:00:01\r\u001b[K     |█████████████████████▊          | 706kB 4.8MB/s eta 0:00:01\r\u001b[K     |██████████████████████          | 716kB 4.8MB/s eta 0:00:01\r\u001b[K     |██████████████████████▍         | 727kB 4.8MB/s eta 0:00:01\r\u001b[K     |██████████████████████▊         | 737kB 4.8MB/s eta 0:00:01\r\u001b[K     |███████████████████████         | 747kB 4.8MB/s eta 0:00:01\r\u001b[K     |███████████████████████▎        | 757kB 4.8MB/s eta 0:00:01\r\u001b[K     |███████████████████████▋        | 768kB 4.8MB/s eta 0:00:01\r\u001b[K     |████████████████████████        | 778kB 4.8MB/s eta 0:00:01\r\u001b[K     |████████████████████████▎       | 788kB 4.8MB/s eta 0:00:01\r\u001b[K     |████████████████████████▋       | 798kB 4.8MB/s eta 0:00:01\r\u001b[K     |█████████████████████████       | 808kB 4.8MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▏      | 819kB 4.8MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▌      | 829kB 4.8MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▉      | 839kB 4.8MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▏     | 849kB 4.8MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▌     | 860kB 4.8MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▉     | 870kB 4.8MB/s eta 0:00:01\r\u001b[K     |███████████████████████████     | 880kB 4.8MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▍    | 890kB 4.8MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▊    | 901kB 4.8MB/s eta 0:00:01\r\u001b[K     |████████████████████████████    | 911kB 4.8MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▍   | 921kB 4.8MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▊   | 931kB 4.8MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████   | 942kB 4.8MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▎  | 952kB 4.8MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▋  | 962kB 4.8MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████  | 972kB 4.8MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▎ | 983kB 4.8MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▋ | 993kB 4.8MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▉ | 1.0MB 4.8MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▏| 1.0MB 4.8MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▌| 1.0MB 4.8MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▉| 1.0MB 4.8MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 1.0MB 4.8MB/s \n","\u001b[?25hInstalling collected packages: sentencepiece\n","Successfully installed sentencepiece-0.1.85\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"QpYr-iTLyvhR","colab_type":"code","outputId":"475cbf1d-e102-4954-d29a-f1c30036bb84","executionInfo":{"status":"ok","timestamp":1585568726826,"user_tz":-480,"elapsed":4506,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}},"colab":{"base_uri":"https://localhost:8080/","height":615}},"source":["!pip install --upgrade tensorflow\n","import tensorflow as tf\n","from tensorflow.keras.layers import Dense, Input\n","from tensorflow.keras.optimizers import Adam\n","from tensorflow.keras.models import Model\n","from tensorflow.keras.callbacks import EarlyStopping\n","import tensorflow_hub as hub\n","import numpy as np"],"execution_count":13,"outputs":[{"output_type":"stream","text":["Requirement already up-to-date: tensorflow in /usr/local/lib/python3.6/dist-packages (2.2.0rc1)\n","Requirement already satisfied, skipping upgrade: gast==0.3.3 in /usr/local/lib/python3.6/dist-packages (from tensorflow) (0.3.3)\n","Requirement already satisfied, skipping upgrade: google-pasta>=0.1.8 in /usr/local/lib/python3.6/dist-packages (from tensorflow) (0.2.0)\n","Requirement already satisfied, skipping upgrade: six>=1.12.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow) (1.12.0)\n","Requirement already satisfied, skipping upgrade: astunparse==1.6.3 in /usr/local/lib/python3.6/dist-packages (from tensorflow) (1.6.3)\n","Requirement already satisfied, skipping upgrade: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow) (1.27.2)\n","Requirement already satisfied, skipping upgrade: numpy<2.0,>=1.16.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow) (1.18.2)\n","Requirement already satisfied, skipping upgrade: scipy==1.4.1; python_version >= \"3\" in /usr/local/lib/python3.6/dist-packages (from tensorflow) (1.4.1)\n","Requirement already satisfied, skipping upgrade: tensorflow-estimator<2.3.0,>=2.2.0rc0 in /usr/local/lib/python3.6/dist-packages (from tensorflow) (2.2.0rc0)\n","Requirement already satisfied, skipping upgrade: wheel>=0.26; python_version >= \"3\" in /usr/local/lib/python3.6/dist-packages (from tensorflow) (0.34.2)\n","Requirement already satisfied, skipping upgrade: h5py<2.11.0,>=2.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow) (2.10.0)\n","Requirement already satisfied, skipping upgrade: opt-einsum>=2.3.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow) (3.2.0)\n","Requirement already satisfied, skipping upgrade: protobuf>=3.8.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow) (3.10.0)\n","Requirement already satisfied, skipping upgrade: tensorboard<2.2.0,>=2.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow) (2.1.1)\n","Requirement already satisfied, skipping upgrade: wrapt>=1.11.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow) (1.12.1)\n","Requirement already satisfied, skipping upgrade: keras-preprocessing>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow) (1.1.0)\n","Requirement already satisfied, skipping upgrade: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow) (1.1.0)\n","Requirement already satisfied, skipping upgrade: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow) (0.9.0)\n","Requirement already satisfied, skipping upgrade: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.8.0->tensorflow) (46.0.0)\n","Requirement already satisfied, skipping upgrade: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.2.0,>=2.1.0->tensorflow) (0.4.1)\n","Requirement already satisfied, skipping upgrade: requests<3,>=2.21.0 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.2.0,>=2.1.0->tensorflow) (2.21.0)\n","Requirement already satisfied, skipping upgrade: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.2.0,>=2.1.0->tensorflow) (1.0.0)\n","Requirement already satisfied, skipping upgrade: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.2.0,>=2.1.0->tensorflow) (3.2.1)\n","Requirement already satisfied, skipping upgrade: google-auth<2,>=1.6.3 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.2.0,>=2.1.0->tensorflow) (1.7.2)\n","Requirement already satisfied, skipping upgrade: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.2.0,>=2.1.0->tensorflow) (1.3.0)\n","Requirement already satisfied, skipping upgrade: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard<2.2.0,>=2.1.0->tensorflow) (3.0.4)\n","Requirement already satisfied, skipping upgrade: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard<2.2.0,>=2.1.0->tensorflow) (2019.11.28)\n","Requirement already satisfied, skipping upgrade: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard<2.2.0,>=2.1.0->tensorflow) (1.24.3)\n","Requirement already satisfied, skipping upgrade: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard<2.2.0,>=2.1.0->tensorflow) (2.8)\n","Requirement already satisfied, skipping upgrade: cachetools<3.2,>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tensorboard<2.2.0,>=2.1.0->tensorflow) (3.1.1)\n","Requirement already satisfied, skipping upgrade: rsa<4.1,>=3.1.4 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tensorboard<2.2.0,>=2.1.0->tensorflow) (4.0)\n","Requirement already satisfied, skipping upgrade: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tensorboard<2.2.0,>=2.1.0->tensorflow) (0.2.8)\n","Requirement already satisfied, skipping upgrade: oauthlib>=3.0.0 in /usr/local/lib/python3.6/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.2.0,>=2.1.0->tensorflow) (3.1.0)\n","Requirement already satisfied, skipping upgrade: pyasn1>=0.1.3 in /usr/local/lib/python3.6/dist-packages (from rsa<4.1,>=3.1.4->google-auth<2,>=1.6.3->tensorboard<2.2.0,>=2.1.0->tensorflow) (0.4.8)\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"s11gvUsLmjV5","colab_type":"text"},"source":["## 4.2 BERT模型"]},{"cell_type":"code","metadata":{"id":"G9lxKW_SmpXL","colab_type":"code","colab":{}},"source":["module_url=\"https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/1\"\n","bert_layer = hub.KerasLayer(module_url, trainable=True)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"cXoUWhm11a1Z","colab_type":"code","colab":{}},"source":["#bert of tf-hub\n","vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()\n","do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()\n","tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"Lxag3s_sz32k","colab_type":"code","outputId":"525e9433-fbbc-4bd6-aa46-42c5b237082c","executionInfo":{"status":"ok","timestamp":1585390710710,"user_tz":-480,"elapsed":28663,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}},"colab":{"base_uri":"https://localhost:8080/","height":425}},"source":["train_input = text_encode(X_train.astype(str), tokenizer, max_len=64)\n","test_input = text_encode(X_test.astype(str), tokenizer, max_len=64)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["75788     underwear, almost embarrassed hard was. almost...\n","26057      love aj make \"getting smoke\" white fuck 😂😂😂 #raw\n","56632     —with thought ruining perfectly settled night....\n","49526     awwww little kiddo :( it’s ok live 21st c vide...\n","119302    taste!!! already listen joji yaeji i'll defini...\n","                                ...                        \n","114980    “know makes sick, mitt?” reflection mirror, mi...\n","90187     wow, lobsterman making really good points. ......\n","80733     oh don’t even get started bernie bros. barely ...\n","103469                    thought house.. ask selling.. lol\n","487       loves life!! i’m lucky dog!! kisses hugs thank...\n","Name: clean_tweet, Length: 32000, dtype: object\n","75231     ceremony done- warning, shiro mistaken. he, ac...\n","51862     poor people would spend money eating dead fish...\n","107143    life death situation yes get priority person l...\n","32730     wasn’t following you! sorted now. best luck bu...\n","66199     thats stupid ass logic nigga im gonna cap, poi...\n","                                ...                        \n","28211     imagine 20 years watching classic games young ...\n","106716     please don’t put child phone me. lord jesus pliz\n","75976     keith cries lot first, used feeling affection....\n","83797     forward way, deal we’ve got, better real. rest...\n","91989     they're 8 year old group constantly changed so...\n","Name: clean_tweet, Length: 8000, dtype: object\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"CAVnxXz50wuG","colab_type":"code","outputId":"49240621-93a5-4a52-eee9-bea372718029","executionInfo":{"status":"ok","timestamp":1585390712021,"user_tz":-480,"elapsed":29903,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}},"colab":{"base_uri":"https://localhost:8080/","height":391}},"source":["model = build_model(bert_layer, max_len=64)\n","model.summary()"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Model: \"model\"\n","__________________________________________________________________________________________________\n","Layer (type)                    Output Shape         Param #     Connected to                     \n","==================================================================================================\n","input_word_ids (InputLayer)     [(None, 64)]         0                                            \n","__________________________________________________________________________________________________\n","input_mask (InputLayer)         [(None, 64)]         0                                            \n","__________________________________________________________________________________________________\n","segment_ids (InputLayer)        [(None, 64)]         0                                            \n","__________________________________________________________________________________________________\n","keras_layer (KerasLayer)        [(None, 768), (None, 109482241   input_word_ids[0][0]             \n","                                                                 input_mask[0][0]                 \n","                                                                 segment_ids[0][0]                \n","__________________________________________________________________________________________________\n","tf_op_layer_strided_slice (Tens [(None, 768)]        0           keras_layer[0][1]                \n","__________________________________________________________________________________________________\n","dense (Dense)                   (None, 2)            1538        tf_op_layer_strided_slice[0][0]  \n","==================================================================================================\n","Total params: 109,483,779\n","Trainable params: 109,483,778\n","Non-trainable params: 1\n","__________________________________________________________________________________________________\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"4Zgak3Gh03sU","colab_type":"code","outputId":"c2cefee8-9cfb-42e4-ea8d-116308d82039","executionInfo":{"status":"ok","timestamp":1585393568194,"user_tz":-480,"elapsed":5014,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}},"colab":{"base_uri":"https://localhost:8080/","height":173}},"source":["train_history = model.fit(\n","    train_input, y_train,\n","    epochs=6,\n","    batch_size=16,\n","    validation_split=0.1,callbacks=[EarlyStopping(monitor='val_loss',patience=2,restore_best_weights=True)]\n",")"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Epoch 1/6\n","1800/1800 [==============================] - 547s 304ms/step - loss: 0.4823 - sparse_categorical_accuracy: 0.7650 - val_loss: 0.4316 - val_sparse_categorical_accuracy: 0.8050\n","Epoch 2/6\n","1800/1800 [==============================] - 545s 303ms/step - loss: 0.3599 - sparse_categorical_accuracy: 0.8449 - val_loss: 0.4063 - val_sparse_categorical_accuracy: 0.8184\n","Epoch 3/6\n","1800/1800 [==============================] - 546s 303ms/step - loss: 0.2884 - sparse_categorical_accuracy: 0.8847 - val_loss: 0.4251 - val_sparse_categorical_accuracy: 0.8200\n","Epoch 4/6\n","1800/1800 [==============================] - 547s 304ms/step - loss: 0.2092 - sparse_categorical_accuracy: 0.9231 - val_loss: 0.4463 - val_sparse_categorical_accuracy: 0.8216\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"Pzq3T-xtAZIb","colab_type":"code","colab":{}},"source":["model.save_weights(path+'bert_model_epoch5.h5')"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"yVM0uCd00-da","colab_type":"code","outputId":"c68bcf17-8405-449b-b820-1d72eeeb20b4","executionInfo":{"status":"ok","timestamp":1585393568201,"user_tz":-480,"elapsed":61,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}},"colab":{"base_uri":"https://localhost:8080/","height":88}},"source":["test_pred = model.predict(test_input)\n","predictions = np.argmax(test_pred, axis=-1)\n","print(\"bert accuracy_score:\",accuracy_score(y_test, predictions))"],"execution_count":0,"outputs":[{"output_type":"stream","text":["WARNING:tensorflow:5 out of the last 6 calls to <function recreate_function.<locals>.restored_function_body at 0x7f1f57eda510> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n"],"name":"stdout"},{"output_type":"stream","text":["WARNING:tensorflow:5 out of the last 6 calls to <function recreate_function.<locals>.restored_function_body at 0x7f1f57eda510> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n"],"name":"stderr"},{"output_type":"stream","text":["bert accuracy_score: 0.838375\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"5cxzdttnDR_7","colab_type":"code","outputId":"6d40f339-bdbc-4699-f24b-96d91136b19b","executionInfo":{"status":"ok","timestamp":1585393568203,"user_tz":-480,"elapsed":50,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"source":["print(\"bert accuracy_score:\",accuracy_score(y_test, predictions))"],"execution_count":0,"outputs":[{"output_type":"stream","text":["bert accuracy_score: 0.838375\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"zsKKRGic2lJK","colab_type":"text"},"source":["## 4.3 ALBERT模型"]},{"cell_type":"code","metadata":{"id":"JXuO3ViP2kMR","colab_type":"code","colab":{}},"source":["module_url=\"https://tfhub.dev/tensorflow/albert_en_xlarge/1\"\n","albert_layer = hub.KerasLayer(module_url, trainable=True)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"bQiT3wX05jZn","colab_type":"code","colab":{}},"source":["#en albert of tf-hub\n","sp_model_file = albert_layer.resolved_object.sp_model_file.asset_path.numpy()\n","tokenizer = tokenization.FullSentencePieceTokenizer(sp_model_file)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"OMryt2Uw5qI2","colab_type":"code","outputId":"e211d2d9-16c5-40d7-a326-2fda25c34319","executionInfo":{"status":"ok","timestamp":1585448532820,"user_tz":-480,"elapsed":4937,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}},"colab":{"base_uri":"https://localhost:8080/","height":425}},"source":["train_input = text_encode(X_train.astype(str), tokenizer, max_len=64)\n","test_input = text_encode(X_test.astype(str), tokenizer, max_len=64)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["42220      og game inviting tea coffee. already knew happen\n","98998          math spent years worth anarchy tickets seat.\n","90302     site i'm watching debate keeps losing msnbc fe...\n","55227                doesn’t change fact driftor says wrong\n","111926    nah campaign pacing great. exploit weaknesses ...\n","                                ...                        \n","12256     current theory handful humans survived near ex...\n","3581                     \"people's vote\" mean, people agree\n","35880     still think nba allow new york knicks draft zi...\n","129751    answered questions resorted \"joke\" try minimis...\n","64498     i've called russian troll and/or bot dared cri...\n","Name: clean_tweet, Length: 32000, dtype: object\n","111208    strange music bring back many memories. exampl...\n","45850     almost 3 years trump self dealing, obstruction...\n","9131      excitement palpable new information brought light\n","119971    got local instead rapid somehow go 2 different...\n","116427    laura pidcock out? that's one saddest losses n...\n","                                ...                        \n","36880               always am, characters share enthusiasm.\n","67189                kuzma nearly good alot yall making be.\n","78213             shcuskfjrnf colors may accurate want fuck\n","119401                cute cb tour dates jjk1 agust2 thinks\n","28223                  fsu guy too? double dip pleasure so.\n","Name: clean_tweet, Length: 8000, dtype: object\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"pwhz_X9I5__P","colab_type":"code","outputId":"ec245cdc-a3a5-43dc-8f3c-c443b769facb","executionInfo":{"status":"ok","timestamp":1585395230696,"user_tz":-480,"elapsed":5936,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}},"colab":{"base_uri":"https://localhost:8080/","height":391}},"source":["model = build_model(albert_layer, max_len=64)\n","model.summary()"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Model: \"model\"\n","__________________________________________________________________________________________________\n","Layer (type)                    Output Shape         Param #     Connected to                     \n","==================================================================================================\n","input_word_ids (InputLayer)     [(None, 64)]         0                                            \n","__________________________________________________________________________________________________\n","input_mask (InputLayer)         [(None, 64)]         0                                            \n","__________________________________________________________________________________________________\n","segment_ids (InputLayer)        [(None, 64)]         0                                            \n","__________________________________________________________________________________________________\n","keras_layer (KerasLayer)        [(None, 2048), (None 58724864    input_word_ids[0][0]             \n","                                                                 input_mask[0][0]                 \n","                                                                 segment_ids[0][0]                \n","__________________________________________________________________________________________________\n","tf_op_layer_strided_slice (Tens [(None, 2048)]       0           keras_layer[0][1]                \n","__________________________________________________________________________________________________\n","dense (Dense)                   (None, 2)            4098        tf_op_layer_strided_slice[0][0]  \n","==================================================================================================\n","Total params: 58,728,962\n","Trainable params: 58,728,962\n","Non-trainable params: 0\n","__________________________________________________________________________________________________\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"u5XcEiP86Eh2","colab_type":"code","outputId":"98d55488-4007-4034-90b2-9b20e9c7e871","executionInfo":{"status":"error","timestamp":1585412518850,"user_tz":-480,"elapsed":2920810,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}},"colab":{"base_uri":"https://localhost:8080/","height":518}},"source":["train_history = model.fit(\n","    train_input, y_train,\n","    epochs=6,\n","    batch_size=16,\n","    validation_split=0.1,callbacks=[EarlyStopping(monitor='val_loss',patience=2,restore_best_weights=True)]\n",")"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Epoch 1/6\n","1800/1800 [==============================] - 5710s 3s/step - loss: 0.5382 - sparse_categorical_accuracy: 0.7249 - val_loss: 0.4397 - val_sparse_categorical_accuracy: 0.7975\n","Epoch 2/6\n","1800/1800 [==============================] - 5768s 3s/step - loss: 0.4043 - sparse_categorical_accuracy: 0.8257 - val_loss: 0.3994 - val_sparse_categorical_accuracy: 0.8334\n","Epoch 3/6\n","1800/1800 [==============================] - 5745s 3s/step - loss: 0.3605 - sparse_categorical_accuracy: 0.8511 - val_loss: 0.3657 - val_sparse_categorical_accuracy: 0.8472\n","Epoch 4/6\n","   5/1800 [..............................] - ETA: 1:13:54 - loss: 0.4019 - sparse_categorical_accuracy: 0.8375"],"name":"stdout"},{"output_type":"error","ename":"KeyboardInterrupt","evalue":"ignored","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)","\u001b[0;32m<ipython-input-20-4355bc6ae2ae>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m6\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m16\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m     \u001b[0mvalidation_split\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mEarlyStopping\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmonitor\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'val_loss'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mpatience\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mrestore_best_weights\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m )\n","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36m_method_wrapper\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m     63\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_method_wrapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     64\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_in_multi_worker_mode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 65\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     66\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m     \u001b[0;31m# Running inside `run_distribute_coordinator` already.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing, **kwargs)\u001b[0m\n\u001b[1;32m    781\u001b[0m                 batch_size=batch_size):\n\u001b[1;32m    782\u001b[0m               \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_train_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 783\u001b[0;31m               \u001b[0mtmp_logs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    784\u001b[0m               \u001b[0;31m# Catch OutOfRangeError for Datasets of unknown size.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    785\u001b[0m               \u001b[0;31m# This blocks until the batch has finished executing.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    578\u001b[0m         \u001b[0mxla_context\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mExit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    579\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 580\u001b[0;31m       \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    581\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    582\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mtracing_count\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    609\u001b[0m       \u001b[0;31m# In this case we have created variables on the first call, so we run the\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    610\u001b[0m       \u001b[0;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 611\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=not-callable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    612\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    613\u001b[0m       \u001b[0;31m# Release the lock early so that multiple threads can perform the call\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   2418\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2419\u001b[0m       \u001b[0mgraph_function\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_maybe_define_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2420\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mgraph_function\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_filtered_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2421\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2422\u001b[0m   \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_filtered_call\u001b[0;34m(self, args, kwargs)\u001b[0m\n\u001b[1;32m   1663\u001b[0m          if isinstance(t, (ops.Tensor,\n\u001b[1;32m   1664\u001b[0m                            resource_variable_ops.BaseResourceVariable))),\n\u001b[0;32m-> 1665\u001b[0;31m         self.captured_inputs)\n\u001b[0m\u001b[1;32m   1666\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1667\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_call_flat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcaptured_inputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcancellation_manager\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1744\u001b[0m       \u001b[0;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1745\u001b[0m       return self._build_call_outputs(self._inference_function.call(\n\u001b[0;32m-> 1746\u001b[0;31m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0m\u001b[1;32m   1747\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n\u001b[1;32m   1748\u001b[0m         \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m    596\u001b[0m               \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    597\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mattrs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 598\u001b[0;31m               ctx=ctx)\n\u001b[0m\u001b[1;32m    599\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    600\u001b[0m           outputs = execute.execute_with_cancellation(\n","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     58\u001b[0m     \u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0;32m---> 60\u001b[0;31m                                         inputs, attrs, num_outputs)\n\u001b[0m\u001b[1;32m     61\u001b[0m   \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mKeyboardInterrupt\u001b[0m: "]}]},{"cell_type":"code","metadata":{"id":"DGDkM59a7mXU","colab_type":"code","outputId":"2fa1212b-2480-4839-b335-018bd3c7ae36","executionInfo":{"status":"ok","timestamp":1585412947091,"user_tz":-480,"elapsed":426559,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}},"colab":{"base_uri":"https://localhost:8080/","height":88}},"source":["model.save_weights(path+'albert_model_epoch5.h5')\n","test_pred = model.predict(test_input)\n","predictions = np.argmax(test_pred, axis=-1)\n","print(\"bert accuracy_score:\",accuracy_score(y_test, predictions))"],"execution_count":0,"outputs":[{"output_type":"stream","text":["WARNING:tensorflow:5 out of the last 6 calls to <function recreate_function.<locals>.restored_function_body at 0x7ff721acc730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n"],"name":"stdout"},{"output_type":"stream","text":["WARNING:tensorflow:5 out of the last 6 calls to <function recreate_function.<locals>.restored_function_body at 0x7ff721acc730> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n"],"name":"stderr"},{"output_type":"stream","text":["bert accuracy_score: 0.845\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"CZnrvb8HFOcM","colab_type":"code","outputId":"d3854843-9262-43fe-febf-aa765f5b6a99","executionInfo":{"status":"ok","timestamp":1585462132480,"user_tz":-480,"elapsed":3383382,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}},"colab":{"base_uri":"https://localhost:8080/","height":139}},"source":["#时间关系上面的6个epoch只训练了3个就手动终止了，接下来利用保存好的参数继续接着运行3个epoch\n","model_continue = build_model(albert_layer, max_len=64)\n","model_continue.load_weights(path+'albert_model_epoch5.h5')\n","train_history = model_continue.fit(\n","    train_input, y_train,\n","    epochs=3,\n","    batch_size=16,\n","    validation_split=0.1,callbacks=[EarlyStopping(monitor='val_loss',patience=2,restore_best_weights=True)]\n",")   "],"execution_count":0,"outputs":[{"output_type":"stream","text":["Epoch 1/3\n","1800/1800 [==============================] - 4510s 3s/step - loss: 0.3350 - sparse_categorical_accuracy: 0.8652 - val_loss: 0.3450 - val_sparse_categorical_accuracy: 0.8612\n","Epoch 2/3\n","1800/1800 [==============================] - 4520s 3s/step - loss: 0.2960 - sparse_categorical_accuracy: 0.8856 - val_loss: 0.3218 - val_sparse_categorical_accuracy: 0.8691\n","Epoch 3/3\n","1800/1800 [==============================] - 4527s 3s/step - loss: 0.2496 - sparse_categorical_accuracy: 0.9089 - val_loss: 0.3244 - val_sparse_categorical_accuracy: 0.8734\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"p6tSUNnw6NP0","colab_type":"code","outputId":"4c2a9fef-f744-4de2-8a20-3de034d056c6","executionInfo":{"status":"ok","timestamp":1585462615413,"user_tz":-480,"elapsed":427648,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}},"colab":{"base_uri":"https://localhost:8080/","height":88}},"source":["model_continue.save_weights(path+'albert_model_epoch5.h5')\n","test_pred = model_continue.predict(test_input)\n","predictions = np.argmax(test_pred, axis=-1)\n","print(\"bert accuracy_score:\",accuracy_score(y_test, predictions))"],"execution_count":0,"outputs":[{"output_type":"stream","text":["WARNING:tensorflow:5 out of the last 6 calls to <function recreate_function.<locals>.restored_function_body at 0x7f8fe96f2bf8> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n"],"name":"stdout"},{"output_type":"stream","text":["WARNING:tensorflow:5 out of the last 6 calls to <function recreate_function.<locals>.restored_function_body at 0x7f8fe96f2bf8> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n"],"name":"stderr"},{"output_type":"stream","text":["bert accuracy_score: 0.869625\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"-KIAA0vGOa6M","colab_type":"text"},"source":["# 五、模型的融合"]},{"cell_type":"markdown","metadata":{"id":"-1z8QcZ3e8LE","colab_type":"text"},"source":["## 5.1 在真实测试集对模型性能进行测试"]},{"cell_type":"markdown","metadata":{"id":"q6tnHfgGagTw","colab_type":"text"},"source":["在原始数据集上抽取相同数量的测试数据进行测试\n","\n","为什么不用之前模型测试样本进行测试？  \n","答：采用没有进行过预处理的原始tweet进行测试会更加真实"]},{"cell_type":"code","metadata":{"id":"SbPJ2LcvSLGf","colab_type":"code","colab":{}},"source":["from sklearn.metrics import accuracy_score\n","import pandas as pd"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"2c2YptAZUkRY","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":68},"outputId":"471b46a0-6ac3-43c9-9fb8-0110e86c391e","executionInfo":{"status":"ok","timestamp":1585572177890,"user_tz":-480,"elapsed":951,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}}},"source":["df=pd.read_csv(path+\"tweets_0.80.csv\")\n","#正负样本均采样2000条进行测试\n","data_pos=df.loc[df.target==4].sample(n=2000, random_state=2)\n","data_neg=df.loc[df.target==0].sample(n=2000, random_state=2)\n","#合并正负样本\n","data=pd.concat([data_pos,data_neg])\n","#删除第一列\n","data.drop(columns=data.columns[0],inplace=True)\n","#删除多余列\n","data.drop(columns=data.columns[1:-1],inplace=True)\n","#正向标签设置为1\n","data.loc[data['target'] ==4, 'target'] = 1\n","print(data.target.value_counts())"],"execution_count":33,"outputs":[{"output_type":"stream","text":["1    2000\n","0    2000\n","Name: target, dtype: int64\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"vAqRnc_eUUG7","colab_type":"code","colab":{}},"source":["#划分训练集和测试集\n","from sklearn.model_selection import train_test_split\n","X_train, X_test, y_train, y_test = train_test_split( data[\"tweet\"], data[\"target\"], test_size=0.2, random_state=42)"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"RsJQsh4DPjm2","colab_type":"text"},"source":["模型1"]},{"cell_type":"code","metadata":{"id":"nDBkx4rhOaMt","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":442},"outputId":"eb336f23-1662-4d56-c301-2b0c87169289","executionInfo":{"status":"ok","timestamp":1585572282840,"user_tz":-480,"elapsed":13353,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}}},"source":["#bert_layer\n","module_url=\"https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/1\"\n","bert_layer = hub.KerasLayer(module_url, trainable=True)\n","#tokenizer\n","vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()\n","do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()\n","tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case)\n","#文本输入编码\n","train_input = text_encode(X_train.astype(str), tokenizer, max_len=64)\n","test_input = text_encode(X_test.astype(str), tokenizer, max_len=64)\n","#模型性能展示\n","model_1 = build_model(bert_layer, max_len=64)\n","model_1.load_weights(path+'bert_model_epoch5.h5')\n","model_1_pred = model_1.predict(test_input)\n","predictions = np.argmax(model_1_pred, axis=-1)\n","print(\"bert accuracy_score:\",accuracy_score(y_test, predictions))"],"execution_count":35,"outputs":[{"output_type":"stream","text":["9036      The tone deafness of the GOP is aweing.  \\n\\nI...\n","124170    Well let’s not act like this is a reach. Could...\n","45857     Yup. \\n\\n\"Hearing views I don't agree with is ...\n","27602     With Vue going away soon, checking out Youtube...\n","90795     Explain the difference between run offense and...\n","                                ...                        \n","37994     So as citizens, what do we do to ensure a fair...\n","17062     Her friends Craig and Frank (who no doubt wear...\n","112818    If they’re being truthful about not connecting...\n","65700     \"This version\" was the same version we played ...\n","114019    We will destroy the environment because people...\n","Name: tweet, Length: 3200, dtype: object\n","123408    ⠀⠀❝This has to be a first time...\\n\\nIl y a un...\n","69799     I don't care how ANYONE  pronounces it, as lon...\n","91306     Nah I’m laughing cause I started ww3 in my men...\n","73193     Who in the IRS is colluding with China? Hillar...\n","18756     After my useless reply re Jamaica, I'm quite s...\n","                                ...                        \n","27787     That's funny, I read it as peach the first tim...\n","52985     How many times has we won that belt again? A h...\n","78605     I'm not brave enough to write this and I'm too...\n","119302    taste!!! i already listen to joji and yaeji bu...\n","73469     Meanwhile, leftist US politicians and media ar...\n","Name: tweet, Length: 800, dtype: object\n","bert accuracy_score: 0.79625\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"5khVO8IAPebf","colab_type":"text"},"source":["模型2"]},{"cell_type":"code","metadata":{"id":"KkCaYa8SPnAd","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":442},"outputId":"661d5dbe-06aa-4ec9-a32c-29d80dca1fe6","executionInfo":{"status":"ok","timestamp":1585572317756,"user_tz":-480,"elapsed":28655,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}}},"source":["#albert_layer\n","module_url=\"https://tfhub.dev/tensorflow/albert_en_xlarge/1\"\n","albert_layer = hub.KerasLayer(module_url, trainable=True)\n","#tokenizer\n","sp_model_file = albert_layer.resolved_object.sp_model_file.asset_path.numpy()\n","tokenizer = tokenization.FullSentencePieceTokenizer(sp_model_file)\n","#文本输入编码\n","train_input = text_encode(X_train.astype(str), tokenizer, max_len=64)\n","test_input = text_encode(X_test.astype(str), tokenizer, max_len=64)\n","#模型性能展示\n","model_2 = build_model(albert_layer, max_len=64)\n","model_2.load_weights(path+'albert_model_epoch5.h5')\n","model_2_pred = model_2.predict(test_input)\n","predictions = np.argmax(model_2_pred, axis=-1)\n","print(\"bert accuracy_score:\",accuracy_score(y_test, predictions))"],"execution_count":36,"outputs":[{"output_type":"stream","text":["9036      The tone deafness of the GOP is aweing.  \\n\\nI...\n","124170    Well let’s not act like this is a reach. Could...\n","45857     Yup. \\n\\n\"Hearing views I don't agree with is ...\n","27602     With Vue going away soon, checking out Youtube...\n","90795     Explain the difference between run offense and...\n","                                ...                        \n","37994     So as citizens, what do we do to ensure a fair...\n","17062     Her friends Craig and Frank (who no doubt wear...\n","112818    If they’re being truthful about not connecting...\n","65700     \"This version\" was the same version we played ...\n","114019    We will destroy the environment because people...\n","Name: tweet, Length: 3200, dtype: object\n","123408    ⠀⠀❝This has to be a first time...\\n\\nIl y a un...\n","69799     I don't care how ANYONE  pronounces it, as lon...\n","91306     Nah I’m laughing cause I started ww3 in my men...\n","73193     Who in the IRS is colluding with China? Hillar...\n","18756     After my useless reply re Jamaica, I'm quite s...\n","                                ...                        \n","27787     That's funny, I read it as peach the first tim...\n","52985     How many times has we won that belt again? A h...\n","78605     I'm not brave enough to write this and I'm too...\n","119302    taste!!! i already listen to joji and yaeji bu...\n","73469     Meanwhile, leftist US politicians and media ar...\n","Name: tweet, Length: 800, dtype: object\n","bert accuracy_score: 0.8425\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"c-9VmfNCXDKF","colab_type":"text"},"source":["现有的结果：  \n","model1: 0.7962  \n","model2: 0.8425"]},{"cell_type":"markdown","metadata":{"id":"jMzk9ZyKXd1X","colab_type":"text"},"source":["## 5.2 两个模型性能均分"]},{"cell_type":"code","metadata":{"id":"SHzEc95tXdMI","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":34},"outputId":"f68c9e25-351d-4ff8-ca57-95e4d99a5065","executionInfo":{"status":"ok","timestamp":1585572350494,"user_tz":-480,"elapsed":517,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}}},"source":["pred_average=(model_1_pred+model_2_pred)/2\n","predictions = np.argmax(pred_average, axis=-1)\n","print(\"average accuracy_score:\",accuracy_score(y_test, predictions))"],"execution_count":37,"outputs":[{"output_type":"stream","text":["average accuracy_score: 0.85125\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"Hv5UMZMoYFKC","colab_type":"text"},"source":["## 5.3 试图找到最好的模型分配比列"]},{"cell_type":"markdown","metadata":{"id":"lc_mmOH_udyi","colab_type":"text"},"source":["\n","2个模型\n","```\n","prob = alpha prob(model1) + (1 - alpha) prob(model2)\n","```\n","\n","5个模型\n","```\n","scores=[]\n","for alpha in np.linspace(0,1,20):\n","    for beta in np.linspace(0,1,20):\n","        for gamma in np.linspace(0,1,20):\n","            for theta in np.linspace(0,1,20):\n","                if(alpha+beta+gamma+theta>1):\n","                    continue\n","                else:\n","                    cm_probabilities = alpha*m+beta*m2+gamma*m3+theta*m4+(1-alpha-beta-gamma-theta)*m5\n","                    cm_predictions = np.argmax(cm_probabilities, axis=-1)\n","                    score = f1_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)), average='weighted')   \n","                    if(score > best_score):\n","                        best_alpha = alpha\n","                        best_beta  = beta\n","                        best_gamma = gamma\n","                        best_theta = theta\n","                        best_score = score\n","    scores.append(best_score)\n","plt.plot(scores)\n","```\n","\n"]},{"cell_type":"code","metadata":{"id":"K2xlpD5VZJEg","colab_type":"code","colab":{}},"source":["import numpy as np\n","from matplotlib import pyplot as plt"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"jgNdXcREYDCO","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":282},"outputId":"e99ea1a9-3a46-4444-ddc9-423022655d03","executionInfo":{"status":"ok","timestamp":1585572357718,"user_tz":-480,"elapsed":1028,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}}},"source":["scores = []\n","for alpha in np.linspace(0,1,100):\n","    pred_probabilities= alpha*model_1_pred+(1-alpha)*model_2_pred\n","    predictions = np.argmax(pred_probabilities, axis=-1)\n","    scores.append(accuracy_score(y_test, predictions))\n","plt.plot(scores)"],"execution_count":39,"outputs":[{"output_type":"execute_result","data":{"text/plain":["[<matplotlib.lines.Line2D at 0x7fa3bb2d9128>]"]},"metadata":{"tags":[]},"execution_count":39},{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deXxU5b3H8c9vJjvZyAJkg4Q9AYTE\nGBAUF1ABRQXtLaCttrbcVsWl2pZaq9bbxVpr696qtfbSXigqKipUXBB3ICRhC4shbFmAkEBICNmf\n+8eM6RASMpCZzGTm93698mLmnGdmfocTvpx5znOeI8YYlFJK+S6LpwtQSinlXhr0Sinl4zTolVLK\nx2nQK6WUj9OgV0opHxfg6QLai4uLM6mpqZ4uQymlepUNGzYcNsbEd7TO64I+NTWV3NxcT5ehlFK9\niojs7Wyddt0opZSP06BXSikfp0GvlFI+ToNeKaV8nAa9Ukr5OA16pZTycRr0Sinl4zToleohxhiW\nbyzjyPFGT5ei/IwGvVI95PNdldyxOJ9b/5lHS6veB0L1HA16pXrIsrxSrBbhi+JKnvuoyNPlKD/i\ndVMgKOWL6hqbWbmlnOuzkqlrauGP73/FhMGxZKfGeLo05Qc06JXqAe9uPUBdYwuzs5JIT4xk4/6j\n3LmkgAdnZmARwWoRJgyOJTTI6ulSlQ/SoFeqByzLKyW5byjnpcZgsQhPzc3kG3/+gvmLNrS1mTgk\nlkW3jMdqEQ9WqnyRBr1SbnbwWD2fFR3mtkuGYrGH+NiUaD756SUcOtYAwNrdlfzqnW38ec0ubrtk\nqCfLVT5Ig14pN3uzoJRWA7Myk05a3j8yhP6RIQCMTopkY0k1j7+3kwmDYzh3kPbdK9fRUTdKdUNT\nS2uXbZbllTIuJZrB8eGdthERfj1rNInRIdyxuIC9lcc5VFPPoZp6jNGhmKp7NOiVOkt/XrOLrP95\nj8KyY522KSw7xvYDNczOSuq0zdciQwJ5am4WB4/Vc9HvPyLn1x+Q8+sP+M7L63XcveoW7bpR6izk\n7qni9+/uoKXVcPviPN5ecAFhQaf+c1qWV0KgVZh5TqJT7zsuJZqlPzi/7T+PvZXHeeGT3dp3r7pF\ng16pM1Rd18SdSwpIig7l51em84N/bOCh5Vt59PqxJ7VrbmnlzY1lXDKiH337BDn9/lkD+5I1sC9g\nmzbhwLEG7btX3aJdN0qdAWMMP31tEweP1fPk3EyuGDWAWy8ewtLcEpZvLDup7adFh6moaXCq26Yz\n7fvuq+uaursJyg/pEb1SXVj0xR6e+2gXrQZajKGipoH7ZoxkXEo0AHdNHc6XxVXct2wz45KjGRgb\nBthOwkaFBnLJyH7d+vyv++6vf+5zJv9+NaGBtouqJg+PO+VbhFId0SN6pU5jw94qHnqrkP5RIVw0\nPJ5LR/Tjp9NG8r0LBre1CbRaeGLOOCwCC5bk09TSSk19E6sKD3DVOQkEB3T/atdxKdE8e0MW00YN\n4KLh8QyMCWNpbgklR+q6/d7K9zl1RC8i04AnACvwojHmkXbrBwJ/B6LtbRYaY1aISCqwDdhhb/ql\nMeYHrildKfeqrmvijsW2vvj//W4OESGBnbZN7hvG7647hx/+M4/HVu1gSHw49U2tzM5Kdlk9l48a\nwOWjBgCwv6qOCx9dzZsFZXqSVnWpy6AXESvwDHAZUAKsF5HlxphCh2b3A0uNMc+JSAawAki1r9tl\njBnn2rKVci9jDAuX2friX/3hxNOG/Nemj0lg3viB/GVNMUnRoaTGhpE1MNot9aXEhJGTGsNreSXc\nevEQRHTaBNU5Z47oc4AiY0wxgIgsAa4BHIPeAJH2x1HAyWellOoF3iwo5Z1N5QAcb2zms6JKfjb9\nP33xznjgqgxy91Sx82Atd08d7tYAnp2VxMJlm9lUUs3YM6hR+R9n+uiTgP0Oz0vsyxw9BNwoIiXY\njuYXOKxLE5F8EVkjIhd29AEiMl9EckUkt6KiwvnqlXKRDXur+NHSjWwprWZfVR2VtY18a8Igvn/h\n4K5f7CAk0MqzN2QxY8wA5uakuKlam+ljEggKsLAsr8Stn6N6P1eNupkLvGyM+YOInA8sEpHRQDkw\n0BhTKSLnAm+IyChjzEmXEhpjngeeB8jOztZLAFWPcuyLf+eOC5zqpjmdof0iePaGc11UXeeiQgO5\nLKM/yzeW8fMrMwgK0LEVqmPO/GaUAo6HJsn2ZY5uAZYCGGO+AEKAOGNMgzGm0r58A7ALGN7dopVy\nFce++CfnZnY75Hva7MwkjtQ1sWanfhNWnXPmiH49MExE0rAF/BxgXrs2+4ApwMsiko4t6CtEJB6o\nMsa0iMhgYBhQ7LLqlTqNNwtKKTly4rRtSo6cYOWWA2fcF+8tJg+PJ7ZPEK9tKOGyjP6eLkd5qS6D\n3hjTLCK3A+9iGzr5kjFmq4g8DOQaY5YD9wAviMjd2E7M3myMMSIyGXhYRJqAVuAHxpgqt22NUnbF\nFbXcuaTAqbbTRw844754bxFotXB9djLPf1zMZ0WHmTQ0ztMlKS8k3jYFanZ2tsnNzfV0GaqX+8Oq\nHTyzuoiPf3IJ/SJCTts20Cq9enhiXWMzM5/6lGP1zay880LiwoM9XZLyABHZYIzJ7midnr1RPqe1\n1fB6fimThsaR3DeMoADLaX96c8gDhAUF8PS8LKpPNHHvKxtp1SmNVTs6141yq6rjjWwprW57PjIh\nossj7O5av6eKkiMnuOdy/znvn54Qyf1XpvPAm1t5bNUOJgyOBaBfZDAjB0R28Wrl6zToldscrWvk\nqic/oay6vm1ZTJ8gVt55Ydst9Nzh9fxSwoKsXGGfLsBffGvCID4rOsyzH+3i2Y92ARBgEXLvn0p0\nmPPTJCvfo0Gv3MIYw09e3URFbQPP3pBF/8hgqk80cds/87n7XwUsumU8Vovru0zqm1p4Z1M500YP\n6PBGIL5MRHhmXhabS6tpNYYtpcd4cPlWCsuPMXGInqT1Z9pHr9ziH1/uZVXhQX46bSQzxiRw7qAY\nLh3Zn19ePYrPd1Xy5zW73PK57xUepKahmetcOJlYbxJgtZA5sC/nDophxpgEALaV13i4KuVp/nXI\no3rEtvJj/M8727h4RDzfnZR20rpvZCfzSdFhHn9vJ5kp0YxKjAIgNMjqkis7X88vZUBkSFsftT+L\njwgmLjyIbeWd39NW+QcNeuVSdY3NLFicT1RoII99YyyWdt0zX98xqWD/Eea9uLZteUJUCB/9+OJu\nzd1+uLaBNTsr+N6FaW7pFuqN0hMi2X5Ag97fadArl3r4rUJ2VdTyj1vGdzqeOzIkkCXzz2fV1gMY\nA2VHT/Dip7tZvb2CaaPP/gTq8oIyWlqN33bbdCQ9IZKXP99Dc0srAVbtqfVXGvTKZd7aWMaS9fu5\n9eIhXV6hmRQdynfs3TrNLa28UVDGsrySbgX9svwSRidFMrx/xFm/h69JT4igsbmV4sPH9e/Fj+l/\n8col9lfVcd+yzWQOjObuy85s/HqA1cK14xJZveMQR443ntXn7zxYw5bSY8zO1KN5R1+Podd+ev+m\nR/QKgH2VdcxflMuJppZT1oUEWHnw6oyThuidaGxhweI8vjpUC8DRuiYQeHJOJoFn0UUwKyuJFz/d\nzdubyvjW+aln/PpleaVYLcLV4xLP+LW+bEh8OIFWYVt5Ddfofd78lga9AmBV4QG2H6hh5thErO3O\nY27Yd4Q7Fuez4s4L265q/eVbW/lg+yFmjEkg0GKbK+ab56WQEhN2Vp+fkRDJyAERLMsvPeOgb2k1\nvFlQykXD43Wel3aCAiwM7RehR/R+ToNeAbB2dxWpsWE8NTfzlHU7D9Zw9dOfcs/Sjfz9Ozm8s7mc\nJev388OLh/DTaSNd8vkiwqzMJH67cjvFFbUMjg93+rVfFldSXl3PfTPSXVKLr0lPiODTrw57ugzl\nQdpHr2htNazfU0VOWkyH64f3j+CBq0bxyVeHefjtwra++B+dYV98V67NTMIi8EZ++/vanN6yvFIi\nggN0PvZOpA+I5FBNA5W1DZ4uRXmIHtH3kJr6Jl74uJhvnZ9KfIR3dS/sPFTD0bomxqd1fpHR3JwU\nPis6zMuf7yEiJOCs++JPp39kCJOGxvGPtfvYXVkHQL+IYO69fAShQR2Pr6+pb+LfW8qZOTaRkMCz\nH4Pvy9ITvj4hW8MFw7zrd0/1DD2i7yFLc0t48sMi7v5XgddNI7tut+1eMJ0d0YOta+U3s8cwZWQ/\nnpgz7qz74rsyf/JgYvoEsbW0mi2l1fz10908/HZhp+0feHMrJ5pamDd+oFvq8QXpCbZhlXrhlP/S\nI/oesiyvhIjgAD4tOsyfP97FrRcP9XRJbdYWV5EYFUJy39DTtosKDeSvN5/n1louHBbP+z+6qO35\nb1ds4y8fF3PB0DiuPCfhpLavbSjh9fxS7po6jHOSe99tAHtKbHgw/SKCKdQTsn5Lj+h7wI4DNWwt\nO8bdlw3nyjEJ/GHVTvL2HfF0WYBtlsm1u6sYPzjWK2/Acc/lIxibEs3CZZvYX1XXtry4opZfvLmF\nnLQYFlw6zIMV9g7pCZE6uZkf85kj+kM19byZX9ZlO4tFmHlOAv3azYe+YW8VeXuPtj2fNDSOjETX\n3LBhWX5J2xjv685NpmD/Ue5YnM9NXQwjFIEZYxJIjD79kXZ3FB8+zuHahtN223hSUICFp+ZkcuWT\nn3Db/+Ux8xzbOPnX8koICrDwxJxxOq+NE0YmRPD5rsM0Nre6ZPI41bv4TNCXH63n1yu2OdX21Q0l\nvH7rxLaTd1tKq5n7/FoaW1rb2kSEBLDijgu73Rfd0mp4I7+Uix3GeD81L5Ob/rrOqXr/b90+3l5w\ngdvmVnemf97TBsaG8ch153D3vwrYVGL7OwsOsPDMvCwSotz3n6AvOW9QDH9ZU8zLn+9m/uQhni5H\n9TCfCfrRSVFs+eUVXbb7vOgw8xdt4JGV23no6lEcb7DNtti3TyCv3zqJyNBADlTXM+uZz7hjST5L\n//v8bo0u+WJXJQePNfDAVf+5ND9rYF9yfzGVppbTn5TdsPcIN/9tHQ8t38qj14896xpOZ93uKuLC\ngxkc18ct7+8qV56TwJT0fjTbT2QHWERH2ZyBKen9uGJUfx799w7Gp8UyNkXPafgTn/kOZ7UI4cEB\nXf5cPmoA352Uxsuf7+G9woM88OZW9lQe50/fzCQxOpTw4ACG9gvnt9eNIX/fUf743s5u1bUsr4SI\nkACmpPc7aXlwgLXLWi8aHs+tFw9haW4Jyzd23S11powxrC2uZHxajFf2z7cXEvifvzMN+TMjIjx6\n3Vj6R4awYHE+NfVNni5J9SCfOaI/Ez+dPoK1uytZsDiP+qZW7pgyjPOHnDyG/KpzEvn0q8M8t2YX\nqXF9GGTvwkmL73PKza1bWg0F+4/S7ND1A9BiDCu3HODazLMf433X1OF8sauS+5ZtZlxyNANjO+9K\nOtHYQl1jM7FOTgNQcuQEZdX1/GCw93bbKNeJCgvkiTnj+ObzX7Lwtc18+/xBAISHBLTdAMYZh2rq\niQ4N0r7+XsQvgz44wMpTczOZ+dSnnJfalzsu7Xio44MzR7Fh7xF+8uqmtmWRIQG849B3b4zh3lc2\n8vpprua8/tyzn1Ex0GrhybmZzHjiExYszuOVH0zs8B9YY3Mr33z+C/ZX1bHyzskMiOr65tv/XLsP\nQO/G5EeyU2O4e+owHlu1k3c2l7ct//mMdL4/eXCXry+uqGXmU58yNiXabff9Va4nxnjXxTvZ2dkm\nNze3Rz7r4LF6okIDT3u0XdvQzKb9ttE4dY0t3P2vAob1D+df9r77VzeUcO8rG7nlgjSmjOx3yuvD\nQwJcMsZ75eZyfvjPPP77osH8bPqpc7r86u1CXvx0N0EBFjJTovm/70847T/Cz4oOc+Nf1/LN7BQe\nue6cbteneg9jDJtKqjne0AzAS5/tZs3OCl774cTT/q42NLcw65nP2VVRS0NzKz++YgS3XeI914P4\nOxHZYIzJ7midU9+9RGSaiOwQkSIRWdjB+oEislpE8kVkk4jM6GB9rYjce3ab4B79I0O67FIJDw5g\n4tA4Jg6NY2pGf34zewx59r77XRW1/OKNLYxPi+G+Gelt7Rx/XHUhz/QxCcwbP5C/rCnm450VJ61b\nvf0QL366m2+fP4jfzBrD2t1VPLO6qNP3OlzbwF3/KmBIfDgPzhzlkvpU7yEijE2JbvsdfewbY4kP\nD+6y7/63K7ZTWH6MZ+ZlMXNsIo+/t5MNe6t6sHJ1troMehGxAs8A04EMYK6IZLRrdj+w1BiTCcwB\nnm23/nFgZffL9byZYxP5ZnYKz63Zxc1/W0dIoIUn5mT2yFfYB67KYHj/cH60tICyoyeob2qh5Egd\n97yykZEDIrhvRjrXZSVx7bhE/vT+Tr7YVUl9U8tJPycaW7j3lY1Un2ji6XmZnc4ho/xHdFgQT8zN\nZH9VHb94Ywsdfct/v/AgL3++h5snpjI1oz+/njWaxOgQ7lhcQHWdntj1ds700ecARcaYYgARWQJc\nAzhOQGKAr68uigLahoiIyLXAbuC4Kwr2Bg9enUHu3ip2VRznxW9nO9Uf7gohgVaenpfFzKc+ZeIj\nH7YtDw208vS8zLZvJ7+aNYb8/UeZ+8KXnb7X/1wzqu3uQ0qdlxrDXVOH8/h7O7lgWPxJ55UOVNfz\n41c3kpEQyc9m2KaljgwJ5Km5WVz/3OcsXLaJZ2/I6hUjt/xVl330InI9MM0Y8z37828B440xtzu0\nSQBWAX2BPsBUY8wGEQkH3gMuA+4Fao0xj3XwGfOB+QADBw48d+/eva7YNrcqO3qCHQdruGTEqf3y\n7pa37whri//zlXnC4BgyB/Y9qc3+qjpWbC6no/nTEqNDuHpsov7DVCdpaTXc8OKXbCqp5q0FFzAk\nPpyWVsO8F75kc+l/ljn685pdPLJyO7+eNZobxg/yUOUKTt9H76qg/5H9vf4gIucDfwVGA48C64wx\nS0XkIToJekc9eTJWKXWyA9X1TH/iYxKiQnn9ton8ZU0xj7+3k99ffw7fyE45pX1rq+Gmv61j3e4q\nlt9+ASMG6A3IPaW7J2NLAcc9nGxf5ugWYCmAMeYLIASIA8YDj4rIHuAu4D4RuR2llFcaEBXC768f\nS2H5Mf570Qb+9P5OrhmX2OkQYYtFePy/xhEREsiCxXmcaDz1nsPK85wJ+vXAMBFJE5EgbCdbl7dr\nsw+YAiAi6diCvsIYc6ExJtUYkwr8CfiNMeZpl1WvlHK5qRn9+c6kVD7aUUFKTBi/unb0abv54iOC\nefy/xrLzYC2PrdrRg5UqZ3V5MtYY02w/Cn8XsAIvGWO2isjDQK4xZjlwD/CCiNyN7cTszcbbBugr\npZy2cPpIQgKtXDsuiYiQwC7bTx4ez/TRA3hnUzn3X5mu53+8jFNXxhpjVgAr2i17wOFxITCpi/d4\n6CzqU0p5QHCA9Yxv/D5xSCwrtxyg5MgJt92BTJ0dnaxCKeUSOfZ7Dn9ZXOnhSlR7GvRKKZcY1i+c\nvmGBbfc4UN5Dg14p5RIWi3Beagzr9mjQexsNeqWUy+SkxbC3so4D1fWeLkU50KBXSrnM11Ner92t\n/fTeRINeKeUy6QmRhAcHaD+9l9GgV0q5jNUiZKf2Za0GvVfRoFdKudT4tFiKDtVyuLbB06UoOw16\npZRL5aTZ7kG8Xo/qvYZf3jNWKeU+Y5KiCA208u7WAx3eqyG5bxjxEc7dwF65hga9UsqlggIsnJcW\nwxsFZbxRUHbK+vDgAJbfPonB7ea2V+6jQa+Ucrk/fGMsW8qqT1ne1NzKT17bxILF+Sy7dSLBAXor\ny56gQa+Ucrn4iOBO774mInz/f3P53codPDCz/e2nlTvoyVilVI+6LKM/N09M5aXPdvPh9oOeLscv\naNArpXrcwukjSU+I5MevbKKxudXT5fg8DXqlVI8LCbRy55ShVB5vZHPpUU+X4/M06JVSHnFeqm28\nvV5F634a9Eopj4gND2ZYv3DWFmvQu5sGvVLKY3LSYtiw9wjNLdpP704a9Eopjxk/OJbahma2ldd4\nuhSfpkGvlPKYnLZ+ep2/3p006JVSHjMgKoRBsWF6QtbNNOiVUh41Pi2G9XuqaG01ni7FZ2nQK6U8\nKictlqN1Tew8pP307uJU0IvINBHZISJFIrKwg/UDRWS1iOSLyCYRmWFfniMiBfafjSIyy9UboJTq\n3cbb56/X2w+6T5dBLyJW4BlgOpABzBWR9jMR3Q8sNcZkAnOAZ+3LtwDZxphxwDTgLyKiE6kppdok\n9w0lMSpE++ndyJkj+hygyBhTbIxpBJYA17RrY4BI++MooAzAGFNnjGm2Lw+xt1NKqTYiQk5aDGuL\nqzBGI8IdnAn6JGC/w/MS+zJHDwE3ikgJsAJY8PUKERkvIluBzcAPHIIfhzbzRSRXRHIrKirOcBOU\nUr1ddmoMh2sb2F91wtOl+CRXnYydC7xsjEkGZgCLRMQCYIxZa4wZBZwH/ExETrm3mDHmeWNMtjEm\nOz4+3kUlKaV6i1GJtg6BwvJjHq7ENzkT9KVAisPzZPsyR7cASwGMMV9g66aJc2xgjNkG1AKjz7ZY\npZRvGjEgAhHYfkCD3h2cCfr1wDARSRORIGwnW5e3a7MPmAIgIunYgr7C/poA+/JBwEhgj4tqV0r5\niLCgANJi+7BNj+jdossRMMaYZhG5HXgXsAIvGWO2isjDQK4xZjlwD/CCiNyN7YTrzcYYIyIXAAtF\npAloBW41xhx229YopXqt9IRINpeeep9Z1X1ODXU0xqzAdpLVcdkDDo8LgUkdvG4RsKibNSql/MDI\nARG8s7mcmvomIkICPV2OT9ErY5VSXiE9wXZCdudBvULW1TTolVJeIb1t5I0Gvatp0CulvEJiVAiR\nIQF6QtYNNOiVUl5BRBiZEKlB7wYa9Eopr5GREMmOAzU6ZbGLadArpbxGekIEdY0t7Kuq83QpPkWD\nXinlNb4eeaPdN66lQa+U8hrD+0dgEQ16V9OgV0p5jZBAK2lxfdh2QIdYupIGvVLKq6TryBuX07s9\nKaW8SnpCJG9vKufBN7cgIoQFWVlw6TBCg6yeLq3X0qBXSnmVycPiefnzPbyeX4oxUNPQzMiESK4e\nm+jp0notDXqllFcZkxzF+p9PBaC5pZVxD7/Hut2VGvTdoH30SimvFWC1cO6gvqzTG4d3iwa9Usqr\n5aTFsPNgLVXHGz1dSq+lQa+U8moTBscA6FF9N2jQK6W82pikaIIDLBr03aBBr5TyakEBFrIG9mXt\n7kpPl9JradArpbze+MExFJYf41h9U9uyytoGig7VUnSolr2VxzFGZ7zsjA6vVEp5vZy0GIyBDXuO\ncMnIfqzbXcWNL66lsaW1rc13JqXy4MxRHqzSe2nQK6W8XtbAvgRahS93V5I5MJq7luSTEB3Cjy4b\njoiwZkcFf/tsD+cPjuXyUQM8Xa7X0aBXSnm9kEArY5OjWVtcxe6K41TUNvDaDydyTnI0AFeM6s/O\ngzX85LVNjE6KIjE61MMVexfto1dK9Qo5aTEU7D/KqsKD/HTayLaQBwgOsPLU3Eyamlu5a0kBzQ5d\nOkqDXinVS4wfHAvAxSPi+e6ktFPWp8b14VezRrNuTxVjHlrFqAf+zagH/s3zH+/q6VK9jnbdKKV6\nhUlDYrn/ynSuy0rGYpEO28zKTKapxbDTPp/9B9sP8UZ+GfMnD+nJUr2OU0EvItOAJwAr8KIx5pF2\n6wcCfwei7W0WGmNWiMhlwCNAENAI/NgY86EL61dK+YkAq4XvXTi4y3b/lZ3S9jgiJJA/fbCT6hNN\nRIUGurM8r9Zl142IWIFngOlABjBXRDLaNbsfWGqMyQTmAM/alx8GZhpjxgA3AYtcVbhSSnWlbVjm\nXv++qtaZPvocoMgYU2yMaQSWANe0a2OASPvjKKAMwBiTb4wpsy/fCoSKSHD3y1ZKqa5lDowmyGph\nbbEGfVeSgP0Oz0vsyxw9BNwoIiXACmBBB+9zHZBnjGlov0JE5otIrojkVlRUOFW4Ukp1JSTQytiU\nKNb6+Tw5rhp1Mxd42RiTDMwAFolI23uLyCjgd8B/d/RiY8zzxphsY0x2fHy8i0pSSilb982W0mqO\nNzR7uhSPcSboS4EUh+fJ9mWObgGWAhhjvgBCgDgAEUkGXge+bYzRcU5KqR6VkxZLc6shb98RT5fi\nMc4E/XpgmIikiUgQtpOty9u12QdMARCRdGxBXyEi0cA72EbhfOa6spVSyjnnDuqL1SJ+Pc1xl0Fv\njGkGbgfeBbZhG12zVUQeFpGr7c3uAb4vIhuBxcDNxjaV3O3AUOABESmw//Rzy5YopVQHwoMDGJ0Y\n6dcnZJ0aR2+MWYHtJKvjsgccHhcCkzp43a+AX3WzRqWU6pactBj+/vle6ptaCAm0erqcHqdTICil\nfN74tFgaW1rZuP+op0vxCA16pZTPOy81BhH8dpilBr1SyudFhQUyon+E396OUINeKeUXJgyOJW/v\nUZr8cApjDXqllF/ISYvhRFMLm0urPV1Kj9OgV0r5hZy0GAC/HE+vQa+U8gtx4cEMie/D2mL/66fX\noFdK+Y2ctFhy9xyhpdV4upQepUGvlPIbEwbHUNPQzLbyY54upUdp0Cul/MZ5qbZ+en8bT69Br5Ty\nG4nRoaTEhLLOz8bTa9ArpfzK+LRY1u2uotWP+uk16JVSfiUnLYYjdU0UVdR6upQeo0GvlPIrE9Ji\nAfxqmKUGvVLKr6TEhDIgMoQv/eiErAa9UsqviAiTh8fx0fZDnGhs8XQ5PUKDXinld67NTOJ4Ywur\nCg94upQeoUGvlPI7E9JiSYwKYVleqadL6REa9Eopv2OxCLOykvjkqwoO1dR7uhy306BXSvmlWZnJ\ntBpYXlDm6VLcToNeKeWXhvYLZ2xylF9032jQK6X81uysZArLj7H9gG9PcqZBr5TyWzPHJhJgEV73\n8aN6DXqllN+K6RPERcPjeWdzuadLcSungl5EponIDhEpEpGFHawfKCKrRSRfRDaJyAz78lj78loR\nedrVxSulVHddMCyOkiMnKD16wtOluE2XQS8iVuAZYDqQAcwVkYx2ze4HlhpjMoE5wLP25fXAL4B7\nXVaxUkq50Hj73De+PHWxMxD0yeYAAAsWSURBVEf0OUCRMabYGNMILAGuadfGAJH2x1FAGYAx5rgx\n5lNsga+UUl5nxIAIIkMCfPqm4QFOtEkC9js8LwHGt2vzELBKRBYAfYCpLqlOKaXczGoRzkuNYW2x\n7wa9q07GzgVeNsYkAzOARSLi9HuLyHwRyRWR3IqKCheVpJRSzhk/OIbiw8d99ipZZ8K4FEhxeJ5s\nX+boFmApgDHmCyAEiHO2CGPM88aYbGNMdnx8vLMvU0opl8hp66f3zaN6Z4J+PTBMRNJEJAjbydbl\n7drsA6YAiEg6tqDXQ3OlVK8wKjGSsCCrzwZ9l330xphmEbkdeBewAi8ZY7aKyMNArjFmOXAP8IKI\n3I3txOzNxhgDICJ7sJ2oDRKRa4HLjTGF7tkcpZQ6c4FWC+cO6uu/QQ9gjFkBrGi37AGHx4XApE5e\nm9qN+pRSqkeMT4vhsVU7OXK8kb59gjxdjkvplbFKKcV/+unX7/G9o3oNeqWUAsamRBEUYGGtD3bf\naNArpRQQHGAlMyWatT54hawGvVJK2U0eHs+W0mPsr6rzdCkupUGvlFJ214xLBOCNfN+atliDXiml\n7JL7hjE+LYZl+aXYR4j7BA16pZRycF1WMrsPHyd//1FPl+IyGvRKKeVg+pgBBAdYfOquUxr0Sinl\nICIkkMtHDeCtTWU0Nrd6uhyX0KBXSql2ZmcmcbSuidU7Dnm6FJfQoFdKqXYuHBZHXHgQy/JKPF2K\nS2jQK6VUOwFWC9eOS+L9bYco8IGTshr0SinVgQWXDmNAZAh3LM6npr7J0+V0iwa9Ukp1ICoskCfm\njKP06Al+/vqWXj2uXoNeKaU6kZ0aw11ThrF8YxmL1+3naF0jR+saqW9q8XRpZ0S87X+p7Oxsk5ub\n6+kylFIKgJZWww0vfsmXDjcPDwm08ME9F5MUHerByk4mIhuMMdkdrXPqxiNKKeWvrBbhLzdm89am\nMppaWmlsbuW3K7fzel4Jt186zNPlOUWDXimluhAVFsiNEwa1Pf9w+yGW5ZVy2yVDEREPVuYc7aNX\nSqkzdF1WMsWHj7OxpNrTpThFg14ppc7Q1/Ph9JYLqjTolVLqDH09H87yjb1jPhwNeqWUOguzs2zz\n4XzUC+bD0aBXSqmzcOHQOOLCg1nWC6Yz1qBXSqmzEGC1cM24RD7cfoijdY2eLue0NOiVUuoszcpM\norGllbc3lXu6lNNyKuhFZJqI7BCRIhFZ2MH6gSKyWkTyRWSTiMxwWPcz++t2iMgVrixeKaU8aVRi\nJCP6R/C6l99MvMugFxEr8AwwHcgA5opIRrtm9wNLjTGZwBzgWftrM+zPRwHTgGft76eUUr2eiDAr\nK4kNe4+w5/BxT5fTKWeO6HOAImNMsTGmEVgCXNOujQEi7Y+jgDL742uAJcaYBmPMbqDI/n5KKeUT\nrh2XhAgs8+KjemeCPgnY7/C8xL7M0UPAjSJSAqwAFpzBaxGR+SKSKyK5FRUVTpaulFKeNyAqhElD\n4ng9v8RrpzJ21cnYucDLxphkYAawSEScfm9jzPPGmGxjTHZ8fLyLSlJKqZ4xOyuJ/VUnyN17xNOl\ndMiZMC4FUhyeJ9uXOboFWApgjPkCCAHinHytUkr1aleMGkBYkNVrx9Q7E/TrgWEikiYiQdhOri5v\n12YfMAVARNKxBX2Fvd0cEQkWkTRgGLDOVcUrpZQ36BMcwLRRA3h7U9kZ3ZSksrahR6ZQ6DLojTHN\nwO3Au8A2bKNrtorIwyJytb3ZPcD3RWQjsBi42dhsxXakXwj8G7jNGNO7bs2ilFJOmJ2VTE19Mx9s\nc25KhOKKWiY/upoHl29xc2V6hymllHKJllbDxEc+YExSFC/edN5p2zY0tzD72c/ZWnaMsCArufdP\nJSyoe7cHOd0dpvTKWKWUcgGrRbg2M4mPdlRQWdtw2ra/W7mDrWXHmD95MHWNLby79YBba9OgV0op\nF5mdmUxzq+GtjWWdtvlw+0Fe+mw3N50/iIXTRpLcN9TtJ3E16JVSykVGDIhgVGJkpxdPHTxWz72v\nbCI9IZKfzUjHYhFmZybxWdFhDh6rd1tdGvRKKeVCs7OS2VRSTdGhmpOWt7Qa7lpSwInGFp6el0lI\noG02mFlZybQaeLPAfUf1GvRKKeVCV49NxGqRU7pjnvuoiC+KK/nlNaMYEh/etjwtrg+ZA6Pd2n2j\nQa+UUi4UHxHM5GFxvJFfSmurbVTjhr1V/PH9r7h6bCLfODf5lNfMzkpm+4EaCsuOuaUmDXqllHKx\nWVnJlFXXM+XxNVz2+Bpufmk9SdGh/HrWaETklPZXjUkg0Cpuu9l49wZuKqWUOsXlGf2ZN35g252n\nMhIjufXioUSEBHbYvm+fIG4YP4ikvqFuqUcvmFJKKR+gF0wppZQf06BXSikfp0GvlFI+ToNeKaV8\nnAa9Ukr5OA16pZTycRr0Sinl4zTolVLKx3ndBVMiUgHs7cZbxAGHXVROb+GP2wz+ud26zf7jTLd7\nkDEmvqMVXhf03SUiuZ1dHear/HGbwT+3W7fZf7hyu7XrRimlfJwGvVJK+ThfDPrnPV2AB/jjNoN/\nbrdus/9w2Xb7XB+9Ukqpk/niEb1SSikHGvRKKeXjfCboRWSaiOwQkSIRWejpetxBRFJEZLWIFIrI\nVhG50748RkTeE5Gv7H/29XSt7iAiVhHJF5G37c/TRGStfZ//S0SCPF2jK4lItIi8KiLbRWSbiJzv\nD/taRO62/35vEZHFIhLii/taRF4SkUMissVhWYf7V2yetG//JhHJOpPP8omgFxEr8AwwHcgA5opI\nhmercotm4B5jTAYwAbjNvp0LgQ+MMcOAD+zPfdGdwDaH578D/miMGQocAW7xSFXu8wTwb2PMSGAs\ntm336X0tIknAHUC2MWY0YAXm4Jv7+mVgWrtlne3f6cAw+8984Lkz+SCfCHogBygyxhQbYxqBJcA1\nHq7J5Ywx5caYPPvjGmz/8JOwbevf7c3+DlzrmQrdR0SSgSuBF+3PBbgUeNXexKe2W0SigMnAXwGM\nMY3GmKP4wb7Gdi/rUBEJAMKAcnxwXxtjPgaq2i3ubP9eA/yvsfkSiBaRBGc/y1eCPgnY7/C8xL7M\nZ4lIKpAJrAX6G2PK7asOAP09VJY7/Qn4CdBqfx4LHDXGNNuf+9o+TwMqgL/Zu6teFJE++Pi+NsaU\nAo8B+7AFfDWwAd/e144627/dyjhfCXq/IiLhwGvAXcaYY47rjG28rE+NmRWRq4BDxpgNnq6lBwUA\nWcBzxphM4Djtuml8dF/3xXb0mgYkAn04tXvDL7hy//pK0JcCKQ7Pk+3LfI6IBGIL+X8aY5bZFx/8\n+muc/c9DnqrPTSYBV4vIHmzdcpdi67+Otn+9B9/b5yVAiTFmrf35q9iC39f39VRgtzGmwhjTBCzD\ntv99eV876mz/divjfCXo1wPD7Gfmg7CdvFnu4Zpczt4v/VdgmzHmcYdVy4Gb7I9vAt7s6drcyRjz\nM2NMsjEmFdu+/dAYcwOwGrje3synttsYcwDYLyIj7IumAIX4+L7G1mUzQUTC7L/vX2+3z+7rdjrb\nv8uBb9tH30wAqh26eLpmjPGJH2AGsBPYBfzc0/W4aRsvwPZVbhNQYP+Zga2/+gPgK+B9IMbTtbrx\n7+Bi4G3748HAOqAIeAUI9nR9Lt7WcUCufX+/AfT1h30N/BLYDmwBFgHBvrivgcXYzkM0YfsGd0tn\n+xcQbCMLdwGbsY1KcvqzdAoEpZTycb7SdaOUUqoTGvRKKeXjNOiVUsrHadArpZSP06BXSikfp0Gv\nlFI+ToNeKaV83P8D6VPt1q8ltScAAAAASUVORK5CYII=\n","text/plain":["<Figure size 432x288 with 1 Axes>"]},"metadata":{"tags":[]}}]},{"cell_type":"code","metadata":{"id":"2Ui9i9MMXNi4","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":34},"outputId":"e0f8e8ef-3d51-4bf4-d89a-8e1a423cece6","executionInfo":{"status":"ok","timestamp":1585572364265,"user_tz":-480,"elapsed":904,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}}},"source":["best_alpha = np.argmax(scores)/100\n","print(best_alpha)"],"execution_count":40,"outputs":[{"output_type":"stream","text":["0.55\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"7Bljqnn9aO0h","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":34},"outputId":"54c33d80-f1b6-4a19-9716-7874d7b84cd8","executionInfo":{"status":"ok","timestamp":1585572369947,"user_tz":-480,"elapsed":480,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}}},"source":["best_pred_probabilities= best_alpha*model_1_pred+(1-best_alpha)*model_2_pred\n","predictions = np.argmax(best_pred_probabilities, axis=-1)\n","print(\"best accuracy_score:\",accuracy_score(y_test, predictions))"],"execution_count":41,"outputs":[{"output_type":"stream","text":["best accuracy_score: 0.85375\n"],"name":"stdout"}]}]}